#region ... Copyright Notice ...
/*
   Copyright 2006-2008 Provo Labs    http://www.provolabs.com

	Author: Tyler Jensen    http://www.tsjensen.com/blog
	Released by the author with written permission of Provo Labs

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 */
#endregion

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Text;

namespace Atrax.Html
{
	public class HtmlDocument
	{
		#region Constructors
		public HtmlDocument(char[] url, char[] html)
		{
			this.url = url;
			this.html = html;
			Parse();
		}

		public HtmlDocument(string url, char[] html)
		{
			this.url = new char[url.Length];
			url.CopyTo(0, this.url, 0, url.Length);
			Parse();
		}

		public HtmlDocument(string url, byte[] html)
		{
			this.html = new char[UTF8Encoding.UTF8.GetCharCount(html)];
			this.html = UTF8Encoding.UTF8.GetChars(html);
			this.url = new char[url.Length];
			url.CopyTo(0, this.url, 0, url.Length);
			Parse();
		}

		public HtmlDocument(string urlString, string htmlString)
		{
			this.url = new char[urlString.Length];
			urlString.CopyTo(0, this.url, 0, urlString.Length);
			this.html = new char[htmlString.Length];
			htmlString.CopyTo(0, this.html, 0, htmlString.Length);
			Parse();
		}
		#endregion

		/// <summary>
		/// Provides public access to the tag list collection.
		/// </summary>
		public HtmlTagCollection Tags { get { return tags; } }

		ExtractedContent m_extract = null;
		public ExtractedContent ExtractedPage 
		{ 
			get 
			{
				if (m_extract == null)
				{
					ContentExtractor ce = new ContentExtractor();
					m_extract = ce.Extract(this, new string(this.url));
				}
				return m_extract; 
			} 
		}

		/// <summary>
		/// Returns a collection of attribute name keys and their values. Attributes with no value such as nowrap
		/// will be returned with the value the same as the key. 
		/// </summary>
		/// <param name="tag"></param>
		/// <returns></returns>
		public Dictionary<string, string> GetAttributes(HtmlTag tag)
		{
			Dictionary<string, string> retval = new Dictionary<string, string>();
			try
			{
				if (tag.EndTag || tag.Element == DtdElement.CDATA || tag.Element == DtdElement.PCDATA) return retval; //no attributes in these cases

				string text = this.ReadSlice(tag.Slice).Trim(tagTrims).Replace("\r", "").Replace("\n", ""); //remove < > / and trailing space
				if (text.Length == 0) return retval;

				int elen = tag.Element.ToString().Length + 1;
				if (elen > text.Length) return retval; //situation of xhtml tag like <BR /> or empty tag like <TR>

				char[] ctxt = text.ToCharArray();

				//replace any space char between two "" or '' with a temp character (char)1
				char[] s1 = new char[1] { (char)1 };
				int i = 0;
				while (i < ctxt.Length)
				{
					if (ctxt[i] == sq) //find next sq
					{
						i++;
						while (i < ctxt.Length && ctxt[i] != sq)
						{
							if (ctxt[i] == ' ') ctxt[i] = s1[0];
							i++;
						}
						if (i >= ctxt.Length) break;
					}
					else if (ctxt[i] == dq) //find next sq
					{
						i++;
						while (i < ctxt.Length && ctxt[i] != dq)
						{
							if (ctxt[i] == ' ') ctxt[i] = s1[0];
							i++;
						}
						if (i >= ctxt.Length) break;
					}
					i++;
				}
				text = new string(ctxt); //has s1 char inside values as space
				text = text.Substring(elen, text.Length - elen).Trim();
				if (text.Length == 0) return retval; //nothing in the tag

				string[] attribs = text.Split(' ');
				char[] trimChars = new char[3] { ' ', '\'', '\"' };
				foreach (string att in attribs)
				{
					if (att.Trim().Length > 0)
					{
						//the '=' character is valid in values, so can't use a split here
						int eq = att.IndexOf("=");
						if (eq < 0) //not found -- has no value, such as nowrap
						{
							if(!retval.ContainsKey(att.Trim().ToLower())) retval.Add(att.Trim().ToLower(), att.Trim());
							continue;
						}
						string key = att.Substring(0, eq).Trim().ToLower();
						string value = att.Substring(eq + 1, att.Length - eq - 1).Trim(trimChars).Replace(s1[0], ' '); //trim then put spaces back in
						if(!retval.ContainsKey(key)) retval.Add(key, value);
					}
				}
			}
			catch (Exception e)
			{
				System.Diagnostics.Debug.WriteLine("Error parsing attributes: " + e.ToString());
			}
			return retval;
		}

		/// <summary>
		/// Gets the string representation of the TagSlice pulled from the Html source.
		/// </summary>
		/// <param name="slice"></param>
		/// <returns></returns>
		public string ReadSlice(TagSlice slice)
		{
			if (slice.Index > -1 && slice.Length > 0 && this.html.Length > slice.Index + slice.Length)
				return new string(this.html, slice.Index, slice.Length);
			else
				return string.Empty;
		}

		/// <summary>
		/// Returns the tag that ends or closes the tag at this zerio based index on the collection. Looks ahead for a tag that is 
		/// either the same and endTag=true or the found tag's DtdElement is not contained in found tag's AllowedContent list.
		/// </summary>
		/// <param name="tag"></param>
		/// <returns></returns>
		public HtmlTag GetCloseTag(int index)
		{
			HtmlTag tag = ((Collection<HtmlTag>)tags)[index]; //must cast to get zero based index.
			return GetCloseTag(tag);
		}

		/// <summary>
		/// Returns the tag that ends or closes this tag. Looks ahead for a tag that is either the same and endTag=true or 
		/// the found tag's DtdElement is not contained in found tag's AllowedContent list.
		/// </summary>
		/// <param name="tag"></param>
		/// <returns></returns>
		public HtmlTag GetCloseTag(HtmlTag tag)
		{
			if (tag.EndTag) return tag; //close tag is its own close tag
			UniqueList<DtdElement> cumulativeAllowedContent = new UniqueList<DtdElement>(HtmlAtoms.List[tag.Element].AllowedContent);
			HtmlTag next = tags.Next(tag);
			while (next != null)
			{
				if (next.EndTag && next.Element == tag.Element) 
				{
					break; //this is the tag's close tag so return this one
				}
				if (!cumulativeAllowedContent.Contains(next.Element) && !EmptyPcData(next)) //not allowed in any cumulative allowedcontent
				{
					break; //close not allowed in content of tag so return this a tag that closes tag that does not require an end tag
				}
				cumulativeAllowedContent.AddRange(HtmlAtoms.List[next.Element].AllowedContent); //NOT very efficient -- TODO -- fix later
				next = tags.Next(next); //move next
			}
			return next;			
		}

		/// <summary>
		/// Returns true if the tag is PCDATA and contains only whitespace characters since all HTML tags can be followed by \r\n\t or space
		/// without affecting the functionality of the HTML.
		/// </summary>
		/// <param name="next"></param>
		/// <returns></returns>
		private bool EmptyPcData(HtmlTag tag)
		{
			if (tag.Element != DtdElement.PCDATA) return false;
			string d = this.ReadSlice(tag.Slice).Trim(this.whiteSpaceTrims);
			if (d.Length > 0)
				return false;
			else
				return true;
		}

		/// <summary>
		/// Looks back up the list for a tag whose AllowedContent list contains the tag's DtdElement at this index 
		/// in a zero based cast of the collection.
		/// </summary>
		/// <param name="tag"></param>
		/// <returns></returns>
		public HtmlTag GetContainerTag(int index)
		{
			HtmlTag tag = ((Collection<HtmlTag>)tags)[index]; //must cast to get zero based index.
			return GetContainerTag(tag);
		}

		/// <summary>
		/// Looks back up the list for a tag whose AllowedContent list contains this tag's DtdElement.
		/// </summary>
		/// <param name="tag"></param>
		/// <returns></returns>
		public HtmlTag GetContainerTag(HtmlTag tag)
		{
			HtmlTag prev = tags.Previous(tag);
			while (prev != null)
			{
				if (tag.EndTag)
				{
					if (!prev.EndTag && prev.Element == tag.Element) break; //found the opening tag
				}
				else if (!prev.EndTag && HtmlAtoms.List[prev.Element].AllowedContent.Contains(tag.Element))
				{
					break; //prev can contain tag, so return this one
				}
				prev = tags.Previous(prev);
			}
			return prev;
		}

		/// <summary>
		/// Iterates through the entire list of tags and returns a collection of tags of the specified type.
		/// </summary>
		/// <param name="element"></param>
		/// <returns></returns>
		public Collection<HtmlTag> GetList(DtdElement element)
		{
			Collection<HtmlTag> retval = new Collection<HtmlTag>();
			foreach (HtmlTag tag in tags)
			{
				if (tag.Element == element) retval.Add(tag);
			}
			return retval;
		}

		/// <summary>
		/// Returns a collection of HtmlTag objects contained by this tag.
		/// </summary>
		/// <param name="tag"></param>
		/// <returns></returns>
		public Collection<HtmlTag> GetContainedList(HtmlTag tag)
		{
			Collection<HtmlTag> retval = new Collection<HtmlTag>();
			if (tag != null)
			{
				HtmlTag close = this.GetCloseTag(tag);
				HtmlTag next = this.tags.Next(tag);
				while (close != null && next != null && next.Slice.Index < close.Slice.Index)
				{
					retval.Add(next);
					next = this.tags.Next(next);
				}
			}
			return retval;
		}

		/// <summary>
		/// Returns a tag containing PCDATA text if matchText found in it.
		/// </summary>
		/// <param name="dtdType"></param>
		/// <param name="matchText"></param>
		/// <param name="ignoreCase"></param>
		/// <returns></returns>
		public HtmlTag GetTagContainingText(DtdElement dtdType, string matchText, bool ignoreCase)
		{
			HtmlTag retval = null;
			bool found = false;
			Collection<HtmlTag> tags = this.GetList(dtdType);
			foreach (HtmlTag tag in tags)
			{
				if (!tag.EndTag)	 //if opening tag, then check for pcdata throug the next tag
				{
					HtmlTag next = this.Tags.Next(tag);
					while (next != null && next.Element != tag.Element)
					{
						if (next.Element == DtdElement.PCDATA)
						{
							string slice = this.ReadSlice(next.Slice);
							if (ignoreCase && slice.ToLower().IndexOf(matchText.ToLower()) > -1)
								found = true;
							else if (slice.IndexOf(matchText) > -1)
								found = true;
							if (found) break;
						}
						next = this.Tags.Next(next);
					}
				}
				if (found)
				{
					retval = tag;
					break;
				}
			}
			return retval;
		}

		/// <summary>
		/// Return all tags that contain the matchText in the pcdata content.
		/// </summary>
		/// <param name="dtdType"></param>
		/// <param name="matchText"></param>
		/// <param name="ignoreCase"></param>
		/// <returns></returns>
		public Collection<HtmlTag> GetAllTagsContainingText(DtdElement dtdType, string matchText, bool ignoreCase)
		{
			Collection<HtmlTag> retval = new Collection<HtmlTag>();
			Collection<HtmlTag> tags = this.GetList(dtdType);
			foreach (HtmlTag tag in tags)
			{
				bool found = false;
				if (!tag.EndTag)	 //if opening tag, then check for pcdata throug the next tag
				{
					HtmlTag next = this.Tags.Next(tag);
					while (next != null && next.Element != tag.Element)
					{
						if (next.Element == DtdElement.PCDATA)
						{
							string slice = this.ReadSlice(next.Slice);
							if (ignoreCase && slice.ToLower().IndexOf(matchText.ToLower()) > -1)
								found = true;
							else if (slice.IndexOf(matchText) > -1)
								found = true;
							if (found) break;
						}
						next = this.Tags.Next(next);
					}
				}
				if (found)
				{
					retval.Add(tag);
				}
			}
			return retval;
		}


		/// <summary>
		/// Returns all PCDATA text contained by the tag, ignoring any tags in between.
		/// </summary>
		/// <param name="tag"></param>
		/// <returns></returns>
		public string GetPlainTextEnclosedByTag(HtmlTag tag)
		{
			StringBuilder sb = new StringBuilder();
			HtmlTag next = this.Tags.Next(tag);
			while (next != null && next.Element != tag.Element)
			{
				if (next.Element == DtdElement.PCDATA)
				{
					string slice = this.ReadSlice(next.Slice);
					sb.Append(slice + " ");
				}
				next = this.Tags.Next(next);
			}
			return sb.ToString().Trim(' ', '\t'); //trim out common whitespace
		}

		/// <summary>
		/// Returns the next tag in the collection of the specified DtdElement that is not an EndTag.
		/// </summary>
		/// <param name="currentTag"></param>
		/// <param name="nextDtdElement"></param>
		/// <returns></returns>
		public HtmlTag GetNextOpenTag(HtmlTag currentTag, DtdElement nextDtdElement)
		{
			HtmlTag next = this.Tags.Next(currentTag);
			while(next != null && (next.Element != nextDtdElement || (next.Element == nextDtdElement && next.EndTag)))
			{
				next = this.Tags.Next(next);
			}
			return next;
		}

		/// <summary>
		/// Get the first tag with the specified element.
		/// </summary>
		/// <param name="element"></param>
		/// <returns></returns>
		public HtmlTag GetFirst(DtdElement element)
		{
			HtmlTag retval = null;
			foreach (HtmlTag tag in tags)
			{
				if (tag.Element == element)
				{
					retval = tag;
					break;
				}
			}
			return retval;
		}

		/// <summary>
		/// Creates a generic collection of HtmlTag objects which are really just smart pointers to the data in the htmlchar[].
		/// The idea is to parse only what we have to in order to start using the data in the document. No hierarchical
		/// data parsing is performed. It's just a linear process with a few look ahead caveats:
		/// <para>SCRIPT and STYLE tags and tags that begine with &lt;! such as comments contain CDATA, so parser 
		/// looks ahead for those to deal with those cases.</para>
		/// </summary>
		private void Parse()
		{
			int n = 0; //used to track back to the beginning of plain text or where the parser closed a tag with '>'
			int i = 0; //our iterator through the html char array
			while (i < html.Length)
			{
				if (html[i] == '<' && i + 2 < html.Length) //min of 3 characters for a tag: <i>
				{
					if (i > n) //we have some PCDATA text to gather before adding this HTML tag
					{
						AddText(i, n);
					}
					i = AddTag(i); //read ahead and create tag and add it to the list
					n = i; //advance crumb to iterator which now points to character following the '>' of the tag just added
				}
				else
				{
					i++; //nothing to see or do
				}
			}
			if (i > n) //we have some trailing PCDATA text to gather as the last HtmlTag
			{
				AddText(i, n);
			}
		}

		private int AddTag(int i)
		{
			/* <S or <s or <! will trigger an investigation into whether the tag about to be added is either
			 * a <script>, <style> or entity that begins with <! such as a comment or <!DOCUMENT type tag.
			 * The <script> and <style> must have a closing tag. <!-- may have a < or > anwhere, so close with --> only
			 * and the <!DOCUMENT will have a closing > but may have a > or < inside single or double quote attribute values.
			 */
			int n = i + 1;
			if (html[n] == '!')
				return AddXTag(i);
			else if (html[n] == 'S' || html[n] == 's')
				return AddSTag(i);
			else
				return AddCommonTag(i);
		}

		private int AddCommonTag(int i)
		{
			int n = i + 1;
			bool endTag = false;
			if (html[n] == '/')
			{
				endTag = true;
				n++;
			}
			DtdElement element = GetElement(n);
			int end = this.FindNextCloseOutsideQuotes(i);
			tags.Add(new HtmlTag(element, new TagSlice(i, end - i + 1), endTag));
			i = end;
			return ++i; //return iterator advanced to the next character
		}

		private DtdElement GetElement(int i)
		{
			DtdElement retval = DtdElement.CDATA;
			//look for a space or > 
			int n = i;
			while (n < html.Length)
			{
				if (html[n] == '>' || html[n] == ' ') break;
				n++;
			}
			if (i >= html.Length || 1 > n - 1)
			{
				retval = DtdElement.CDATA; //we have exceeded limits
			}
			else
			{
				string name = new string(html, i, n - i).ToUpper().Trim(' ', '/');
				try
				{
					retval = (DtdElement)Enum.Parse(typeof(DtdElement), name);
				}
				catch
				{
					retval = DtdElement.CDATA; //unknown / not found
				}
			}
			return retval;
		}

		private int AddXTag(int i)
		{
			//<!--[if !IE]> header <![endif]--> new style comment???
			//second and third chars tell us if it's a comment or not
			if (i + 6 < html.Length && html[i + 2] == '-' && html[i + 3] == '-') //has to be long enough and have -- to start
			{
				int n = i + 6;
				while (n < html.Length)
				{
					if (html[n] == '>' && html[n - 1] == '-' && html[n - 2] == '-') 
						break;
					n++;
				}
				tags.Add(new HtmlTag(DtdElement.COMMENT, new TagSlice(i, n - i + 1), false));
				i = n;
			}
			else
			{
				int end = this.FindNextCloseOutsideQuotes(i);
				tags.Add(new HtmlTag(DtdElement.CDATA, new TagSlice(i, end - i + 1), false));
				i = end;
			}
			return ++i; //return iterator advanced to the next character
		}

		private int AddSTag(int i)
		{
			//second and third letters will tell us whether it's a script or style tag: SCR... and STY... are unique in the tag list
			int n2 = i + 2;
			int n3 = i + 3;
			if (n3 >= html.Length) return AddCommonTag(i); //can't be script or style because we're at the end
			if ((html[n2] == 'c' || html[n2] == 'C') && (html[n3] == 'r' || html[n3] == 'R')) return AddThreePartCDATATag(i, DtdElement.SCRIPT);
			if ((html[n2] == 't' || html[n2] == 'T') && (html[n3] == 'y' || html[n3] == 'Y')) return AddThreePartCDATATag(i, DtdElement.STYLE);
			return AddCommonTag(i);
		}

		private int AddThreePartCDATATag(int i, DtdElement element)
		{
			//adding 3 tags here: element, CDATA and element with endTag = true
			int end = this.FindNextCloseOutsideQuotes(i);
			tags.Add(new HtmlTag(element, new TagSlice(i, end - i + 1), false));
			i = end + 1;
			int close = this.FindNextOpenClose(i);
			//ensure we have the closing tag because SCRIPT can often contain HTML in a string literal
			DtdElement closeElement = this.GetElement(close + 2); //check to see if closing tag for element
			while (close + 2 < html.Length && element != closeElement)
			{
				close = this.FindNextOpenClose(close + 2);
				closeElement = this.GetElement(close + 2); 
			}
			if(close > i) //only add CDATA if there is really data -- could be <SCRIPT bbbb></SCRIPT>
				tags.Add(new HtmlTag(DtdElement.CDATA, new TagSlice(i, close - i), false));

			end = this.FindNextCloseOutsideQuotes(close);
			tags.Add(new HtmlTag(element, new TagSlice(close, end - close + 1), true));
			i = end;
			return ++i; //return iterator advanced to the next character
		}

		private int FindNextOpenClose(int startIndex)
		{
			//looking for </ and will return index of the < in that sequence
			int i = startIndex + 1;
			while (i < html.Length)
			{
				if (html[i - 1] == '<' && html[i] == '/')
				{
					i--;
					break;
				}
				i++;
			}
			return i;
		}

		private int FindNextCloseOutsideQuotes(int startIndex)
		{
			//in some cases, a double set will occur such as <tr bgcolor="#CCCCCC""> and that must be considered as one
			int i = startIndex + 1;
			while (i < html.Length)
			{
				if (html[i] == sq) //find closing quote
				{
					i++;
					if (i >= html.Length) break; //get out if we went to the end and didn't find it
					while (i < html.Length && html[i] != sq) i++; //takes i to the char just after the single quote
					if (i >= html.Length) break; //get out if we went to the end and didn't find it
					if (html[i] == sq) i++; // 'text'' is same as 'text' 
				}
				if (html[i] == dq) //find closing quote
				{
					i++;
					if (i >= html.Length) break; //get out if we went to the end and didn't find it
					while (i < html.Length && html[i] != dq) i++; //takes i to the char just after the single quote
					if (i >= html.Length) break; //get out if we went to the end and didn't find it
					if (html[i] == dq) i++; // "text"" is same as "text"
				}
				if (html[i] == '>')
				{
					break; //found end of tag - return
				}
				i++; //move next
			}
			return i;
		}

		private void AddText(int i, int crumb)
		{
			tags.Add(new HtmlTag(DtdElement.PCDATA, new TagSlice(crumb, i - crumb), false));
		}

		private char[] tagTrims = new char[4] { '<', '>', '/', ' ' };
		private char[] whiteSpaceTrims = new char[4] { ' ', '\t', '\r', '\n' };
		private HtmlTagCollection tags = new HtmlTagCollection();
		private readonly char[] url;
		private readonly char[] html;
		private readonly char dq = '\"';
		private readonly char sq = '\'';
	}
}
